
/*
 *  FilterMap.cpp
 *
 *	Copyright (C) 2012
 *	Crane <szcshui@gmail.com>
 *	All Rights Reserved
 *
 *************************************************************************
 *
 *  Description:
 *      This file implements the bloom filter index file memory structre(Map), 
 *		Support data struture serailized(save to/load from files).
 *
 */

#include <iostream>
#include <fstream>
#include <sstream>
#include <time.h>
#include "FilterMap.h"
#include "Utils.h"

//Filter Map Class
FilterMap::FilterMap(int bitArrayLength, bool onlyAccount, const string& fileName)
	:updated(false)
{
	this->bitArrayLength = bitArrayLength;

	//only use account as key or use account + security key
	// or use both fields as key
	this->onlyAccount = onlyAccount;
	currentPath = ExtractDirectory(fileName);
	filterFileName = fileName;
	LoadFromFile(fileName);
}

FilterMap::~FilterMap(void)
{
	SaveData();
	ClearData();
}

//Free memory
void FilterMap::ClearData(void)
{
	for(map<string, BloomFilter*>::iterator itr = filtersMap.begin();
		itr != filtersMap.end(); itr++)
	{
		delete itr->second;
	}

	filtersMap.clear();

	for(map<string, Index*>::iterator itr = indicesCache.begin();
		itr != indicesCache.end(); itr++)
	{
		delete itr->second;
	}

	indicesCache.clear();
}

void FilterMap::SaveData(void)
{
	//only updated map can be save to file
	//
	if (updated)
	{
		updated = !SaveToFile(filterFileName);
	}
}

bool FilterMap::ParseLine(const string& line)
{
	if (line.length() > 2 )
	{
		int pos = line.find('|');
		if (pos > 0)
		{
			string index = line.substr(0, pos);
			BloomFilter *filter = GetBloomFilter(index);
			filter->LoadFromString(line.substr(pos+1));
			//cout << index << endl;
		}
	}
	return false;
}

bool FilterMap::LoadFromFile(const string& fileName)
{
	bool ret = true;

	ifstream inputFile;
	cout << "loading from " << fileName << endl;

	try
	{
		inputFile.open(fileName.c_str());
		if (inputFile.is_open())
		{
			string line;
			while (getline(inputFile, line))
			{
				ParseLine(line);
				//cout << line << endl;
			}
		}
		else
		{
			cout << "loaded failed"  << endl;
		}
	}
	catch(exception &e )
	{
		cerr << e.what() << endl;
		ret =  false;
	}

	inputFile.close();           // close file

	return ret;
}

bool FilterMap::SaveToFile(const string& fileName)
{
	bool ret = true;
	ofstream outputFile;

	try
	{
		outputFile.open(fileName.c_str(), ios_base::trunc);

		for(map<string, BloomFilter*>::iterator itr = filtersMap.begin();
			itr != filtersMap.end(); itr++)
		{
			string name = itr->first.c_str(); 
			outputFile.write(name.c_str(), name.length());
			outputFile.write("|", 1);
			itr->second->WriteToFile(outputFile);
			outputFile.write("\n", 1);
			outputFile.flush();
		}
		
	}
	catch(exception &e )
	{
		cerr << e.what() << endl;
		ret =  false;
	}

	outputFile.close();           // close file

	cout << fileName << " is saved" << endl;

	return ret;
}

BloomFilter * FilterMap::GetBloomFilter(const string& index)
{
	//default false positive
	const double FalsePositive = 0.0001;

	map<string,BloomFilter *>::iterator itr = filtersMap.find(index) ;
	if (itr == filtersMap.end())
	{
		BloomFilter * filter = new BloomFilter(bitArrayLength, FalsePositive);  
		filtersMap.insert(pair<string,BloomFilter *>(index, filter));
		return filter;
	}
	else
	{
		return itr->second;
	}

}

//Build bit array from index files
void FilterMap::RebuildBloomFilterFromPath(
	const string& path, 
	int start,	//start file name index
	int end		//end file name index
	) 
{
	ClearData();

	for(int i=start; i<=end; ++i)
	{
		std::stringstream oss;
		oss << i;

		string str(oss.str());

		unsigned int len = str.length();
		for(unsigned int j=0;j< 4 - len; ++j)
		{
			str = "0" + str;
		}

		str = "file" + str + ".idx";

		cout << str << endl;

		BuildBloomFilterFile(path + str);
	}

	SaveData();
}

//format: AccountNumber:SecurityId|start-offset
bool FilterMap::BuildBloomFilterFile(const string& fileName)
{
	const char sep = onlyAccount ? ':' : '|';
	bool ret = true;

	ifstream inputFile;

	try
	{
		inputFile.open(fileName.c_str());

		BloomFilter * filter = GetBloomFilter(ExtractFilename(fileName));

		string line;
		while (getline(inputFile, line))
		{
			if (line.length() > 2 )
			{
				int pos = line.find(sep);
				if (pos > 0)
				{
					filter->Insert(line.substr(0, pos));
				}
			}

			//cout << line << endl;
		}
	}
	catch(exception &e )
	{
		cerr << e.what() << endl;
		ret =  false;
	}

	inputFile.close();           // close file

	updated = true;
	return ret;
}

//use file list to search
int FilterMap::SearchWithList(const string& fileName)
{
	cout << "search all keys in " << fileName << endl;
#ifdef _CACHE
	cout << "memory cache will be used. " << endl;
#endif

	time_t startTime;
	time ( &startTime );

	//total queries 
	int total = 0;
	//count for falsePostive
	int falsePositive = 0;

	ifstream inputFile;
	try
	{
		inputFile.open(fileName.c_str());
		if (inputFile.is_open())
		{
			string line;
			while (getline(inputFile, line))
			{
				if (!line.empty())
				{
					++total;
					if ((line.size() > 0) && (line[line.size() - 1] == '\r'))
					{
						line.resize(line.size() - 1);
					}
					int ret = Search(line);
					if (ret > 0)
					{
						falsePositive += ret;
					}
				}
			}
		}
		else
		{
			cout << "loaded failed"  << endl;
		}
	}
	catch(exception &e )
	{
		cerr << e.what() << endl;
	}

	time_t endTime;
	time ( &endTime );

	inputFile.close();           // close file

	cout << "total false positive: " << falsePositive << " in total queries: " << total << endl;

	cout << "average each query duration: "  << difftime (endTime,  startTime) / total << endl;

	return falsePositive;
}

//Find key in map
bool FilterMap::Find(const string& key, vector<string>& result)
{
	if (!key.empty())
	{
		for(map<string, BloomFilter*>::iterator itr = filtersMap.begin();
			itr != filtersMap.end(); itr++)
		{
			if (itr->second->IsPresent(key))
			{
				result.push_back(itr->first);
			}
		}
	}

	return result.size() > 0;

}

//to response false positive count
int FilterMap::Search(const string& key)
{
#ifdef _TRACE
	cout << "Find: " << key << endl;
#endif

	vector<string> result;
	bool found = Find(key, result);

	int falsePositive = 0;
	if ( found )
	{
		for(vector<string>::iterator itr = result.begin(); itr != result.end(); itr++)
		{
#ifdef _TRACE
			cout << "\r\nThe key in these index files:\t" << itr->c_str() << endl;
#endif

#ifdef _CACHE
			if (ReadContentFromCache(key, (*itr)) == 0)
#else
			if (ReadContent(key, currentPath + (*itr)) == 0)
#endif
			{
				++falsePositive;
#ifdef _TRACE
				cout << "!! False Positive Happened: can not find any index in file:" << itr->c_str() << endl;
#endif
			}
		}

		result.clear();
	}
	else
	{
		falsePositive = -1; //can not find the key
		cout << "\r\nCan not find key: " << key  << endl;
	}

#ifdef _TRACE
	if (falsePositive > 0)
	{
		cout << "False Positive number is: " << falsePositive << endl;
	}
	cout << "\r\n" <<  endl;
#endif

	return falsePositive;
}

//To get real offset in each index files
int FilterMap::ReadContent(const string& key, const string& fileName)
{
	int count = 0;

	ifstream  inputFile;
	try
	{
		inputFile.open(fileName.c_str());

		if (inputFile.is_open())
		{
			string line;
			while (getline(inputFile, line))
			{
				if (line.find(key) != -1)
				{
					++count;
#ifdef _TRACE
					cout << "\t" << line << endl;
#endif
				}
			}

			inputFile.close();
		}
		else
		{
			cout << "open index file " << fileName << " failed!"  << endl;
		}
	}
	catch(exception &e )
	{
		cerr << e.what() << endl;
	}

	return count;
}


Index *FilterMap::GetCache(const string& fileName)
{
	map<string, Index*>::iterator itr = indicesCache.find(fileName);
	if(itr == indicesCache.end())
	{
		Index * index = new Index(currentPath + fileName, onlyAccount);
		indicesCache.insert(pair<string, Index *>(fileName, index));
		return index;
	}
	else
	{
		return itr->second;
	}
}

int FilterMap::ReadContentFromCache(const string& key, const string& fileName)
{
	Index * index = GetCache(fileName);
	vector<string> result = index->GetOffsetList(key, onlyAccount);
	int count = result.size();
#ifdef _TRACE
	for(vector<string>::iterator itr = result.begin(); itr != result.end(); itr++)
	{
		cout << "\t" << (*itr) << endl;
	}
#endif
	return count;
}